{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write fake accuracies\n",
    "import tensorflow as tf\n",
    "\n",
    "writer = tf.summary.create_file_writer('./graphs')\n",
    "\n",
    "accuracy = [0.1, 0.4, 0.6, 0.8, 0.9, 0.95] # fake accuracy values\n",
    "\n",
    "with writer.as_default():\n",
    "    for step, acc in enumerate(accuracy):\n",
    "        tf.summary.scalar('Accuracy', acc, step) # add summary\n",
    "        writer.flush() # make sure everything is written to disk\n",
    "\n",
    "writer.close() # not really needed, but good habit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize a basic graph\n",
    "\n",
    "\"\"\"\n",
    "As TensorFlow 2.0 generally operates in eager mode, there is often no graph created, and thus no way to\n",
    "visualize it.\n",
    "\n",
    "The following is a possible workaround using @tf.function. \n",
    "\"\"\"\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.ops import summary_ops_v2\n",
    "\n",
    "# Graph\n",
    "a = tf.Variable(2, name='a')\n",
    "b = tf.Variable(3, name='b')\n",
    "\n",
    "@tf.function\n",
    "def graph_to_visualize(a, b):\n",
    "    c = tf.add(a, b, name='Add')\n",
    "\n",
    "# Visualize\n",
    "writer = tf.summary.create_file_writer('./graphs')\n",
    "\n",
    "with writer.as_default():\n",
    "    graph = graph_to_visualize.get_concrete_function(a, b).graph # get graph from function\n",
    "    summary_ops_v2.graph(graph.as_graph_def()) # visualize\n",
    "\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize a giant graph\n",
    "import tensorflow as tf\n",
    "\n",
    "a = tf.Variable(2.0, name='a')\n",
    "b = tf.Variable(3.0, name='b')\n",
    "c = tf.Variable(7.0, name='c')\n",
    "\n",
    "@tf.function\n",
    "def graph_to_visualize(a, b, c):\n",
    "    d = tf.multiply(a, b, name='d-mul')\n",
    "    e = tf.add(b, c, name='e-add')\n",
    "    f = tf.subtract(e, a, name='f-sub')\n",
    "    g = tf.multiply(d, b, name='g-mul')\n",
    "    h = tf.divide(g, d, name='h-div')\n",
    "    i = tf.add(h, f, name='i-add')\n",
    "\n",
    "writer = tf.summary.create_file_writer('./graphs')\n",
    "\n",
    "with writer.as_default():\n",
    "    graph = graph_to_visualize.get_concrete_function(a, b, c).graph # get graph from function\n",
    "    summary_ops_v2.graph(graph.as_graph_def()) # visualize\n",
    "\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize a U-Net generator\n",
    "\"\"\"Using the previous techniques, it is needlessly hard to visualize a Keras model.\n",
    "Instead, use the Keras TensorBoard callback when compiling your model.\n",
    "\n",
    "We will talk more about this later.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a simple normal distribution\n",
    "import tensorflow as tf\n",
    "\n",
    "normal_dist = tf.random.normal(shape=(10, 10, 10, 10, 10), mean=0, stddev=1) # tensor to be logged, shape is irrelevant\n",
    "\n",
    "writer = tf.summary.create_file_writer('./graphs')\n",
    "with writer.as_default():\n",
    "    tf.summary.histogram(\"NormalDistribution\", normal_dist, step=0) # log tensor\n",
    "    writer.flush()\n",
    "\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stack 100 normal distributions\n",
    "import tensorflow as tf\n",
    "\n",
    "writer = tf.summary.create_file_writer('./graphs')\n",
    "\n",
    "with writer.as_default():\n",
    "    for i in range(100):\n",
    "        normal_dist = tf.random.normal(shape=(10, 10, 10, 10, 10), mean=i/50, stddev=1) # tensor to be logged, shape is irrelevant\n",
    "        tf.summary.histogram(\"NormalDistribution\", normal_dist, step=i) # summary that logs tensor\n",
    "\n",
    "        writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize a multimodal distributon\n",
    "import tensorflow as tf\n",
    "\n",
    "writer = tf.summary.create_file_writer('./graphs')\n",
    "\n",
    "with writer.as_default():\n",
    "    for i in range(100):\n",
    "        normal_dist_0 = tf.random.normal(shape=(10000,), mean=-i/20, stddev=0.5)\n",
    "        normal_dist_1 = tf.random.normal(shape=(10000,), mean=i/20, stddev=0.5)\n",
    "        normal_dist_cat = tf.concat([normal_dist_0, normal_dist_1], axis=0) # concatenate both normal dists\n",
    "\n",
    "        tf.summary.histogram(\"MultimodalDistribution\", normal_dist_cat, step=i) # summary that logs tensor\n",
    "\n",
    "writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize some random nosie\n",
    "import tensorflow as tf\n",
    "\n",
    "image = tf.random.uniform((1, 210, 160, 3), 0, 1, dtype=tf.float32) # batch_size, height, width, channels\n",
    "writer = tf.summary.create_file_writer('./graphs')\n",
    "\n",
    "with writer.as_default():\n",
    "    img_summary = tf.summary.image(\"InputImage\", image, step=0)\n",
    "    writer.flush()\n",
    "\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visuaize the first 4 examples of MNIST\n",
    "import tensorflow as tf\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
    "\n",
    "images = x_train[:4].reshape((-1, 28, 28, 1))\n",
    "writer = tf.summary.create_file_writer('./graphs')\n",
    "\n",
    "with writer.as_default():\n",
    "    tf.summary.image(\"TrainingExamples\", images, max_outputs=4, step=0)\n",
    "\n",
    "    writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display some text\n",
    "import tensorflow as tf\n",
    "\n",
    "text = tf.convert_to_tensor('Hello world!')\n",
    "writer = tf.summary.create_file_writer('./graphs')\n",
    "\n",
    "with writer.as_default():\n",
    "    tf.summary.text(\"Text\", text, step=0)\n",
    "\n",
    "    writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merge summaries\n",
    "\"\"\"There is no equivalent to tf.summary.merge in TensorFlow 2.0\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merge all summaries\n",
    "\"\"\"There is no equivalent to tf.summary.merge_all in TensorFlow 2.0\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use the TensorBoard callback in a Keras model\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Conv2D, Flatten, Dense, MaxPooling2D, Dropout\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
    "\n",
    "x_train = x_train.reshape((-1, 28, 28, 1))\n",
    "x_test = x_test.reshape((-1, 28, 28, 1))\n",
    "\n",
    "y_train = tf.keras.utils.to_categorical(y_train, 10)\n",
    "y_test = tf.keras.utils.to_categorical(y_test, 10)\n",
    "\n",
    "model = Sequential([\n",
    "    Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),\n",
    "    Conv2D(64, (3, 3), activation='relu'),\n",
    "    MaxPooling2D((2, 2)),\n",
    "    Dropout(0.25),\n",
    "    Flatten(),\n",
    "    Dense(128), # we will visualize neurons in this layer\n",
    "    Dropout(0.5),\n",
    "    Dense(10, activation='softmax')\n",
    "])\n",
    "\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard('./graphs',\n",
    "    histogram_freq=1, # how often to compute activation and weight histograms\n",
    "    write_graph=True, # visualize the graph\n",
    "    write_grads=True, # visual gradient histogram\n",
    "    write_images=True, # visualize model weights as an image\n",
    "    # embeddings_freq=1, # how often to visualize embeddings\n",
    "    # embeddings_layer_names=['...'], # names of embedding layers to visualize; wouldn't work on this model\n",
    "    update_freq='epoch' # update TensorBoard every epoch\n",
    ")\n",
    "\n",
    "model.compile(loss=tf.keras.losses.categorical_crossentropy,\n",
    "              optimizer=tf.keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=32,\n",
    "          epochs=15,\n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test),\n",
    "          callbacks=[tensorboard_callback])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
